

In [1]:

    
import cv2, numpy as np
from matplotlib import pyplot as plt
%matplotlib inline

Load Parameters



In [2]:

    
from utils import read_json

params = read_json('parameters.json')

RESIZE_X = params['resize']['x']
RESIZE_Y = params['resize']['y']
ITEM_FOLDER = params['item_folder']

Input Data

Image of the bin
List of items in the bin



In [ ]:

    
bin_stamp = '170405145336'
contents = ["Colgate_Toothbrush_4PK","Epsom_Salts","Duct_Tape",
            "Bath_Sponge","Crayons","Burts_Bees_Baby_Wipes"]



In [3]:

    
bin_stamp = '170405145538'
contents = ["glue_sticks","tissue_box","laugh_out_loud_jokes",
            "toilet_brush","expo_eraser","table_cloth"]



In [4]:

    
contents = [s.lower() for s in contents]

Phase 1: Recognition by SIFT Features

Compute Features in Bin



In [5]:

    
from utils import imread_rgb, compute_sift

filename_bin = 'bin/' + bin_stamp + '.png'
image_bin = imread_rgb(filename_bin)
(kp_bin, des_bin) = compute_sift(image_bin)









    



1067 features detected

Match Bin and Items Features



In [6]:

    
from utils import match_items

items = list(contents)
item_d, recognised_items, mask_items = match_items(image_bin, kp_bin, des_bin, items)









    



Item: "Training_items/glue_sticks/glue_sticks_top_01_sift.npy" Good features: 5
Item: "Training_items/tissue_box/tissue_box_top_01_sift.npy" Good features: 3
Item: "Training_items/laugh_out_loud_jokes/laugh_out_loud_jokes_top_01_sift.npy" Good features: 12
Item: "Training_items/toilet_brush/toilet_brush_top_01_sift.npy" Good features: 2
Item: "Training_items/expo_eraser/expo_eraser_top_01_sift.npy" Good features: 26
Item: "Training_items/table_cloth/table_cloth_top_01_sift.npy" Good features: 3

Not recognised items



In [7]:

    
items = [s for s in contents if s not in recognised_items]
items









    Out[7]:





['glue_sticks', 'tissue_box', 'toilet_brush', 'table_cloth']



In [8]:

    
kernel = np.ones((3,3),np.uint8)
mask_items = cv2.dilate(mask_items,kernel,iterations = 5)
plt.imshow(mask_items,cmap='gray'), plt.axis('off');

Phase 2: segment bin by depth into upper / lower levels



In [9]:

    
from utils import imread_gray
filename_bin = 'bin/' + bin_stamp + '.pgm'
image_depth = imread_gray(filename_bin)
plt.imshow(image_depth,cmap='gray'); plt.axis('off');



In [10]:

    
from utils import fill_holes

image_depth = fill_holes(image_depth)
plt.imshow(image_depth,cmap='gray');



In [11]:

    
image_depth = cv2.bitwise_and(image_depth,image_depth,mask=255-mask_items)
pix_depth = image_depth.flatten()
pix_depth = pix_depth[pix_depth>0]
dmin, dmax = (np.min(pix_depth),np.max(pix_depth))
h,b = np.histogram(pix_depth,bins=dmax-dmin)
sh = np.cumsum(h)
b_high = [b_val for b_val,sh_val in zip(b,sh) if sh_val>sh[-1]/2]
dcut = int(b_high[0])
plt.hist(pix_depth,bins=dmax-dmin+1);
hmax = np.around(max(h)+1000,-3)
plt.plot([dcut,dcut],[0,hmax],'r-')
plt.axis([dmin,dmax,0,hmax]);



In [12]:

    
upper_bin = cv2.inRange(image_depth,int(dmin),dcut-1)
kernel = np.ones((3,3),np.uint8)
upper_bin = cv2.dilate(upper_bin,kernel,iterations = 3)
plt.imshow(cv2.bitwise_and(image_bin,image_bin,mask=upper_bin));



In [13]:

    
lower_bin = cv2.inRange(image_depth,dcut,int(dmax))
kernel = np.ones((3,3),np.uint8)
lower_bin = cv2.dilate(lower_bin,kernel,iterations = 3)
plt.imshow(cv2.bitwise_and(image_bin,image_bin,mask=lower_bin));

Cluster colors in upper bin



In [14]:

    
from utils import cluster_colors

positions, weights = cluster_colors(image_bin, upper_bin, items)



In [16]:

    
pos_ok = [(p,w[0][1]) for p,w in zip(positions,weights) if len(w)==1]
pos_unkw = [(p,w) for p,w in zip(positions,weights) if len(w)>1]
it_col = {'table_cloth':(255,0,255),'tissue_box':(0,255,255),'glue_sticks':(255,0,0),'toilet_brush':(255,255,0)}
#it_col = {'burts_bees_baby_wipes':(255,0,255),'bath_sponge':(0,255,255),'duct_tape':(255,0,0)}
image_disp = image_bin.copy()
mask_disp = upper_bin.copy()
for item in items:
    it_pos = [p for p,it in pos_ok if it==item]
    for cnt in it_pos:
        cv2.drawContours(image_disp,cnt,-1,it_col[item],2)
        cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
for pos,w in pos_unkw:
    for cnt in pos:
        cv2.drawContours(image_disp,cnt,-1,(0,0,0),1)
        #cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
plt.imshow(image_disp); plt.axis('off');



In [ ]:

Cluster colors in lower bin



In [18]:

    
positions, weights = cluster_colors(image_bin, lower_bin, items)



In [19]:

    
pos_ok = [(p,w[0][1]) for p,w in zip(positions,weights) if len(w)==1]
pos_unkw = [(p,w) for p,w in zip(positions,weights) if len(w)>1]
it_col = {'table_cloth':(255,0,255),'tissue_box':(0,255,255),'glue_sticks':(255,0,0),'toilet_brush':(255,255,0)}
#it_col = {'burts_bees_baby_wipes':(255,0,255),'bath_sponge':(0,255,255),'duct_tape':(255,0,0)}
image_disp = image_bin.copy()
mask_disp = upper_bin.copy()
for item in items:
    it_pos = [p for p,it in pos_ok if it==item]
    for cnt in it_pos:
        cv2.drawContours(image_disp,cnt,-1,it_col[item],2)
        cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
for pos,w in pos_unkw:
    for cnt in pos:
        cv2.drawContours(image_disp,cnt,-1,(0,0,0),1)
        #cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
plt.imshow(image_disp); plt.axis('off');

Phase 3: Segmentation and Recognition by Color



In [27]:

    
%%time
from sklearn.cluster import KMeans, MiniBatchKMeans

n_cc = 20
image_RGBA = np.dstack((image_bin, 255-mask_bin))
pixels = image_RGBA.reshape((image_RGBA.shape[0] * image_RGBA.shape[1], 4))
filtered_pixels = np.array(filter(lambda x:x[3]==255,pixels))
n, _ = filtered_pixels.shape
pixels_LAB = cv2.cvtColor(filtered_pixels[:,0:3].reshape(1,n,3),cv2.COLOR_RGB2LAB)
pixels_LAB = pixels_LAB.reshape(n,3)
#clt = MiniBatchKMeans(n_clusters = n_cc)
clt = KMeans(n_clusters = n_cc)
clt.fit(pixels_LAB)









    



CPU times: user 3.45 s, sys: 95.9 ms, total: 3.55 s
Wall time: 3.54 s



In [28]:

    
image = cv2.cvtColor(image_bin, cv2.COLOR_RGB2LAB)
(h_bin, w_bin) = image.shape[:2]
pixels = image.reshape((image.shape[0] * image.shape[1], 3))
labels = clt.predict(pixels)
quant = clt.cluster_centers_.astype("uint8")[labels]
quant = quant.reshape((h_bin, w_bin, 3))
quant = cv2.cvtColor(quant, cv2.COLOR_LAB2RGB)
#plt.subplot(121),plt.imshow(cv2.bitwise_and(image_bin,image_bin,mask=255-mask_bin)),plt.title('Original'),plt.axis('off');
#plt.subplot(122),plt.imshow(cv2.bitwise_and(quant,quant,mask=255-mask_bin)),plt.title('%d colors' % n_cc),plt.axis('off');
plt.imshow(cv2.bitwise_and(quant,quant,mask=255-mask_bin)),plt.title('%d colors' % n_cc),plt.axis('off');
bin_cc = clt.cluster_centers_



In [29]:

    
bin_hist, _ = np.histogram(clt.predict(pixels_LAB),bins=range(n_cc+1))
plt.bar(range(n_cc), bin_hist);



In [30]:

    
sort_index = np.argsort(bin_hist)[::-1]
sort_index









    Out[30]:





array([15, 18,  3,  4,  0,  6,  5,  1,  8, 19, 10, 11, 12,  9, 16, 13,  7,
        2, 14, 17])



In [31]:

    
recognised_items









    Out[31]:





['laugh_out_loud_jokes', 'expo_eraser']



In [32]:

    
items = [s for s in contents if s not in recognised_items]
items









    Out[32]:





['glue_sticks', 'tissue_box', 'toilet_brush', 'table_cloth']

While not sort_index is empty



In [33]:

    
positions = []
weights = []
while len(sort_index)>0:
    obj_label = sort_index[0]
    d_other = [np.linalg.norm(bin_cc[obj_label,1:]-bin_cc[other,1:]) for other in sort_index]
    obj_labels = [sort_index[idx] for idx,val in enumerate(d_other) if val<20]
    obj_hist = np.array([bin_hist[obj_l] for obj_l in obj_labels],dtype='float32')
    obj_hist = obj_hist / np.sum(obj_hist)
    sort_index = np.array([x for x in sort_index if x not in obj_labels])
    mask = np.zeros((h_bin, w_bin)).astype('uint8')
    for val_label in obj_labels:
        mask = cv2.bitwise_or( mask, ((labels==val_label).astype('uint8') * 255).reshape((h_bin, w_bin)) )
    mask = cv2.bitwise_and( mask, 255-mask_bin)
    kernel = np.ones((3,3),np.uint8)
    mask = cv2.erode(mask,kernel,iterations = 3)
    mask = cv2.dilate(mask,kernel,iterations = 3)
    #cnt, _ = cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
    cnt, _ = cv2.findContours(mask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
    cnt = sorted(cnt, key=lambda x:cv2.contourArea(x), reverse=True)
    best_cnt = [c for c in cnt if cv2.contourArea(c)>500]
    positions.append(best_cnt)

    best_item = []
    views = ['top_01','top-side_01','top-side_02','bottom_01','bottom-side_01','bottom-side_02']
    for item in items:
        for view in views:
            try:
                filename = ITEM_FOLDER + '/' + item + '/' + item + '_' + view + '_dc.json'
                dc = read_json(filename)
                hist = dc['hist']
                obj_cc = dc['cluster_centers']
                sum_h = 0
                for i in range(5):
                    d_bin_obj = [np.linalg.norm(obj_cc[i]-bin_cc[obj_l,:]) for obj_l in obj_labels]
                    index_min = np.argmin(d_bin_obj)
                    if d_bin_obj[index_min] < 25:
                        sum_h += hist[i] * obj_hist[index_min]
                        # hist[i] is the number of pixels in the image -> count only in rectangle?
                #if sum_h > 0.05:
                if sum_h > 0.1:
                    best_item.append((sum_h,item,view))
            except IOError:
                pass
    best_item_one = []
    for it in items:
        try:
            w = max([bi[0] for bi in best_item if bi[1]==it])
            best_item_one.append((w,it))
        except ValueError:
            pass
    weights.append(best_item_one)



In [34]:

    
weights









    Out[34]:





[[(0.44030166975058621, 'table_cloth')],
 [],
 [(0.12954169557295547, 'glue_sticks'),
  (0.167810297784632, 'tissue_box'),
  (0.1417541365540394, 'toilet_brush'),
  (0.11031379046448783, 'table_cloth')],
 [],
 [],
 [(0.18055268442758618, 'tissue_box')]]



In [35]:

    
pos_ok = [(p,w[0][1]) for p,w in zip(positions,weights) if len(w)==1]
pos_unkw = [(p,w) for p,w in zip(positions,weights) if len(w)>1]

Upper bin items



In [25]:

    
it_col = {'table_cloth':(255,0,255),'tissue_box':(0,255,255),'glue_sticks':(255,0,0),'toilet_brush':(255,255,0)}
#it_col = {'burts_bees_baby_wipes':(255,0,255),'bath_sponge':(0,255,255),'duct_tape':(255,0,0)}
image_disp = image_bin.copy()
mask_disp = mask_bin.copy()
for item in items:
    it_pos = [p for p,it in pos_ok if it==item]
    for cnt in it_pos:
        cv2.drawContours(image_disp,cnt,-1,it_col[item],2)
        cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
for pos,w in pos_unkw:
    for cnt in pos:
        cv2.drawContours(image_disp,cnt,-1,(0,0,0),1)
        #cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
plt.imshow(image_disp); plt.axis('off');

Lower bin items



In [37]:

    
it_col = {'table_cloth':(255,0,255),'tissue_box':(0,255,255),'glue_sticks':(255,0,0),'toilet_brush':(255,255,0)}
#it_col = {'burts_bees_baby_wipes':(255,0,255),'bath_sponge':(0,255,255),'duct_tape':(255,0,0)}
image_disp = image_bin.copy()
mask_disp = mask_bin.copy()
for item in items:
    it_pos = [p for p,it in pos_ok if it==item]
    for cnt in it_pos:
        cv2.drawContours(image_disp,cnt,-1,it_col[item],2)
        cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
for pos,w in pos_unkw:
    for cnt in pos:
        cv2.drawContours(image_disp,cnt,-1,(0,0,0),1)
        #cv2.drawContours(mask_disp,cnt,-1,(255,),-1)
plt.imshow(image_disp); plt.axis('off');



In [30]:

    
# For each known item: compute bounding box of all its contours -> dilate
# For each unknown contour:
# compute overlapping with each possible item
# if non overlapping -> remove item from candidates



In [31]:

    
kernel = np.ones((3,3),np.uint8)
mask_disp = cv2.dilate(mask_disp,kernel,iterations = 5)
plt.imshow(mask_disp,cmap='gray'); plt.axis('off');



In [46]:

    
unkw_mask = np.zeros(mask_bin.shape,dtype='uint8')
for pos,w in pos_unkw:
    for cnt in pos:
        cv2.drawContours(unkw_mask,cnt,-1,(255,),-1)
#unkw_mask = cv2.bitwise_and(unkw_mask,unkw_mask,mask=255-mask_disp)
plt.imshow(unkw_mask,cmap='gray'); plt.axis('off');



In [47]:

    
it_mask = np.zeros(mask_bin.shape,dtype='uint8')
for pos,it in pos_ok:
    for cnt in pos:
        cv2.drawContours(it_mask,cnt,-1,(255,),-1)
plt.imshow(it_mask,cmap='gray'); plt.axis('off');



In [48]:

    
image_disp = image_bin.copy()
for pos,w in pos_unkw:
    print(w)
    for cnt in pos:
        cv2.drawContours(image_disp,cnt,-1,(0,0,0),2)
plt.imshow(image_disp); plt.axis('off');









    



[(0.096659176112086673, 'glue_sticks'), (0.13961579294096801, 'tissue_box'), (0.11201143079411489, 'toilet_brush'), (0.095917014235114686, 'table_cloth')]



In [130]:

    
# intersecting contours?



In [35]:

    
plt.imshow(mask_bin,cmap='gray'), plt.axis('off');



In [ ]:

    
# Draw filled contours in mask, dilate